import pandas as pd
import numpy as np
from sklearn import metrics
from tqdm import tqdm
import warnings

warnings.filterwarnings('ignore')

pd.set_option('max_colwidth', 200)
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)

register = pd.read_csv('../data/user_register_log.txt', sep='\t', header=None,
                       names=['user_id', 'register_day', 'register_type', 'device_type'],
                       dtype={0: np.uint32, 1: np.uint8, 2: np.uint8, 3: np.uint16})

launch = pd.read_csv('../data/app_launch_log.txt', sep='\t', header=None,
                     names=['user_id', 'launch_day'],
                     dtype={0: np.uint32, 1: np.uint8})

create = pd.read_csv('../data/video_create_log.txt', sep='\t', header=None,
                     names=['user_id', 'create_day'],
                     dtype={0: np.uint32, 1: np.uint8})

activity = pd.read_csv('../data/user_activity_log.txt', sep='\t', header=None,
                       names=['user_id', 'act_day', 'page', 'video_id', 'author_id', 'action_type'],
                       dtype={0: np.uint32, 1: np.uint8, 2: np.uint8, 3: np.uint32, 4: np.uint32, 5: np.uint8})


# 滑窗法扩充训练集数据
def split_data(data, columns, start_day, end_day):
    data = data[(data[columns] >= start_day) & (data[columns] <= end_day)]
    return data


def features_addday_list():
    return [0, 1, 2, 3, 4, 5, 12]


def ups():
    return 1


def downs():
    return 18


def get_label_list(start_day, end_day):
    result = split_data(launch, 'launch_day', start_day, end_day)['user_id'].drop_duplicates()
    return pd.Series(result)


up = downs() + 1
down = downs() + 7
data = register.loc[:, ['user_id']]
for label_num in range(len(features_addday_list()) - 1):
    # print(label_num, up + label_num, down + label_num)
    label_list = get_label_list(up + label_num, down + label_num)
    label_name = 'label_' + str(label_num)
    data[label_name] = data['user_id'].isin(label_list).replace({True: 1, False: 0})
# data.to_csv('../data/feature/data_label.csv', index=None)

up = ups()
down = downs()
for feature_num in tqdm(features_addday_list()):

    # 基础变量定义
    feature_start = up + 0
    feature_end = down + feature_num
    print(feature_num, feature_start, feature_end)

    result_data = split_data(register, 'register_day', 1, feature_end)
    feature_data = split_data(register, 'register_day', feature_start, feature_end)

    # 提特征(已经包含设备类型、设备类型)
    # 特征区间最大天数减去注册日期
    result_data['maxday_red_registerday'] = max(feature_data['register_day']) - feature_data['register_day']
    result_data = result_data.fillna(max(feature_data['register_day']))
    del result_data['register_day']

    # 保存结果
    # result_file_name = 'register_feature_'+str(feature_num)+'.csv'
    # result_data.to_csv('../data/feature/'+result_file_name, index=None)

up = ups()
down = downs()

for feature_num in tqdm(features_addday_list()):
    # 基础变量定义
    feature_start = up
    feature_end = down + feature_num
    print(feature_num, feature_start, feature_end)

    result_data = split_data(register, 'register_day', 1, feature_end).loc[:, ['user_id', 'register_day']]
    feature_data = split_data(create, 'create_day', feature_start, feature_end)

    # 提特征
    # 用户创建视频计数
    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='create_day',
                                 aggfunc='count').reset_index().rename(columns={"create_day": 'create_count'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    result_data = result_data.fillna(0)

    # 用户创建视频的 平均/最大/最小日期 与 注册日期/最大时间 的时间差
    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='create_day',
                                 aggfunc='mean').reset_index().rename(columns={"create_day": 'create_mean'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    result_data['createmean_red_register'] = result_data['create_mean'] - result_data['register_day']
    result_data['maxday_red_createmean'] = max(result_data['register_day']) - result_data['create_mean']

    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='create_day',
                                 aggfunc=np.max).reset_index().rename(columns={"create_day": 'create_max'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    result_data['createmax_red_register'] = result_data['create_max'] - result_data['register_day']
    result_data['maxday_red_createmax'] = max(result_data['register_day']) - result_data['create_max']

    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='create_day',
                                 aggfunc=np.min).reset_index().rename(columns={"create_day": 'create_min'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    result_data['createmin_red_register'] = result_data['create_min'] - result_data['register_day']
    result_data['maxday_red_createmin'] = max(result_data['register_day']) - result_data['create_min']
    result_data = result_data.fillna(-1)

    # 创建最大间隔
    result_data['max_red_min_create'] = result_data['create_max'] - result_data['create_min']

    # 最后一天是否有活动
    result_data['create_at_lastday'] = pd.Series(
        result_data['create_max'] == max(feature_data['create_day'])).replace({True: 1, False: 0})

    # 均值/最大/最小 天数处理
    result_data['create_mean'] = max(feature_data['create_day']) - result_data['create_mean']
    result_data['create_max'] = max(feature_data['create_day']) - result_data['create_max']
    result_data['create_min'] = max(feature_data['create_day']) - result_data['create_min']

    # 间隔的 方差/均值
    feature_data_tmp = feature_data.drop_duplicates(['user_id', 'create_day']).sort_values(
        by=['user_id', 'create_day'])
    feature_data_tmp['create_gap'] = np.array(feature_data_tmp['create_day']) - np.array(
        feature_data_tmp.tail(1).append(feature_data_tmp.head(len(feature_data_tmp) - 1))['create_day'])

    feature_tmp = pd.pivot_table(feature_data_tmp, index='user_id', values='create_gap',
                                 aggfunc=(lambda a: np.average(a[1:]))).reset_index().rename(
        columns={"create_gap": 'create_gap_mean'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    feature_tmp = pd.pivot_table(feature_data_tmp, index='user_id', values='create_gap',
                                 aggfunc=(lambda a: np.var(a[1:]))).reset_index().rename(
        columns={"create_gap": 'create_gap_var'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    result_data = result_data.fillna(0)

    # 是否一直连续/连续到结束
    result_data['always_create'] = [1 if i == 1 else 0 for i in result_data['create_gap_mean']]
    tmp = (result_data['create_at_lastday'] == 1).replace({True: 1, False: 0})
    result_data['always_create_atlast'] = tmp * result_data['always_create']
    del tmp

    # 创建日期的 方差/峰度/偏度
    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='create_day',
                                 aggfunc=np.var).reset_index().rename(columns={"create_day": 'create_var'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='create_day',
                                 aggfunc=pd.Series.kurt).reset_index().rename(columns={"create_day": 'create_kurt'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='create_day',
                                 aggfunc=pd.Series.skew).reset_index().rename(columns={"create_day": 'create_skew'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    result_data = result_data.fillna(0)

    # 求一天最大创建数
    feature_data['max_create_in_oneday'] = 0
    feature_tmp = pd.pivot_table(feature_data, index=['user_id', 'create_day'], values='max_create_in_oneday',
                                 aggfunc='count').reset_index()
    feature_tmp = pd.DataFrame(feature_tmp.groupby(['user_id'])['max_create_in_oneday'].max()).reset_index()
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    result_data.fillna(0, inplace=True)
    del result_data['register_day']

    # 保存结果
    # result_file_name = 'create_feature_' + str(feature_num) + '.csv'
    # result_data.to_csv('../data/feature/' + result_file_name, index=None)

up = ups()
down = downs()

for feature_num in tqdm(features_addday_list()):
    # 基础变量定义
    feature_start = up
    feature_end = down + feature_num
    print(feature_num, feature_start, feature_end)

    result_data = split_data(register, 'register_day', 1, feature_end).loc[:, ['user_id', 'register_day']]
    feature_data = split_data(launch, 'launch_day', feature_start, feature_end)

    # 提特征
    # 登录计数/登录率
    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='launch_day',
                                 aggfunc='count').reset_index().rename(columns={"launch_day": 'launch_count'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    distance = (max(feature_data['launch_day']) - min(feature_data['launch_day']))
    result_data['launch_ratio'] = result_data['launch_count'] * 1.0 / distance
    result_data = result_data.fillna(0)

    # 登录的 平均/最大/最小日期 与 注册日期/最大时间 的时间差
    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='launch_day',
                                 aggfunc='mean').reset_index().rename(columns={"launch_day": 'launch_mean'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    result_data['launchmean_red_register'] = result_data['launch_mean'] - result_data['register_day']
    result_data['maxday_red_launchmean'] = max(result_data['register_day']) - result_data['launch_mean']

    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='launch_day',
                                 aggfunc=np.max).reset_index().rename(columns={"launch_day": 'launch_max'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    result_data['launchmax_red_register'] = result_data['launch_max'] - result_data['register_day']
    result_data['maxday_red_launchmax'] = max(result_data['register_day']) - result_data['launch_max']

    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='launch_day',
                                 aggfunc=np.min).reset_index().rename(columns={"launch_day": 'launch_min'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    result_data['maxday_red_launchmin'] = max(result_data['register_day']) - result_data['launch_min']
    result_data = result_data.fillna(-1)

    # 登录最大与最小差
    result_data['max_red_min_launch'] = result_data['launch_max'] - result_data['launch_min']

    # 最后一天是否有活动
    result_data['launch_at_lastday'] = pd.Series(result_data['launch_max'] == max(feature_data['launch_day'])).replace(
        {True: 1, False: 0})

    # 均值/最大/最小 天数处理
    result_data['launch_mean'] = max(feature_data['launch_day']) - result_data['launch_mean']
    result_data['launch_max'] = max(feature_data['launch_day']) - result_data['launch_max']
    result_data['launch_min'] = max(feature_data['launch_day']) - result_data['launch_min']

    # 间隔的 方差/均值/最大
    feature_data_tmp = feature_data.drop_duplicates(['user_id', 'launch_day']).sort_values(by=['user_id', 'launch_day'])
    feature_data_tmp['launch_gap'] = np.array(feature_data_tmp['launch_day']) - np.array(
        feature_data_tmp.tail(1).append(feature_data_tmp.head(len(feature_data_tmp) - 1))['launch_day'])

    feature_tmp = pd.pivot_table(feature_data_tmp, index='user_id', values='launch_gap',
                                 aggfunc=(lambda a: np.average(a[1:]))).reset_index().rename(
        columns={"launch_gap": 'launch_gap_mean'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    feature_tmp = pd.pivot_table(feature_data_tmp, index='user_id', values='launch_gap',
                                 aggfunc=(lambda a: np.var(a[1:]))).reset_index().rename(
        columns={"launch_gap": 'launch_gap_var'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    feature_tmp = pd.pivot_table(feature_data_tmp, index='user_id', values='launch_gap',
                                 aggfunc=(lambda a: np.max(a[1:]))).reset_index().rename(
        columns={"launch_gap": 'launch_gap_max'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    result_data = result_data.fillna(0)

    # 是否一直连续/连续到结束
    result_data['always_launch'] = [1 if i == 1 else 0 for i in result_data['launch_gap_mean']]
    tmp = (result_data['launch_at_lastday'] == 1).replace({True: 1, False: 0})
    result_data['always_launch_atlast'] = tmp * result_data['always_launch']
    del tmp

    # 登录日期的 方差/峰度/偏度
    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='launch_day',
                                 aggfunc=np.var).reset_index().rename(columns={"launch_day": 'launch_var'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='launch_day',
                                 aggfunc=pd.Series.kurt).reset_index().rename(columns={"launch_day": 'launch_kurt'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='launch_day',
                                 aggfunc=pd.Series.skew).reset_index().rename(columns={"launch_day": 'launch_skew'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    result_data = result_data.fillna(0)
    del result_data['register_day']

    # 保存结果
    # result_file_name = 'launch_feature_' + str(feature_num) + '.csv'
    # result_data.to_csv('../data/feature/' + result_file_name, index=None)

up = ups()
down = downs()

for feature_num in tqdm(features_addday_list()):

    # 基础变量定义
    feature_start = up
    feature_end = down + feature_num
    print(feature_num, feature_start, feature_end)

    result_data = split_data(register, 'register_day', 1, feature_end).loc[:, ['user_id', 'register_day']]
    feature_data = split_data(activity, 'act_day', feature_start, feature_end)

    # 提特征
    # 活动计数
    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='act_day',
                                 aggfunc='count').reset_index().rename(columns={"act_day": 'act_count'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    result_data = result_data.fillna(0)

    # 活动的 平均/最大/最小日期 与 注册日期/最大时间 的时间差
    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='act_day',
                                 aggfunc='mean').reset_index().rename(columns={"act_day": 'act_mean'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    result_data['actmean_red_register'] = result_data['act_mean'] - result_data['register_day']
    result_data['maxday_red_actmean'] = max(result_data['register_day']) - result_data['act_mean']

    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='act_day',
                                 aggfunc=np.max).reset_index().rename(columns={"act_day": 'act_max'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    result_data['actmax_red_register'] = result_data['act_max'] - result_data['register_day']
    result_data['maxday_red_actmax'] = max(result_data['register_day']) - result_data['act_max']

    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='act_day',
                                 aggfunc=np.min).reset_index().rename(columns={"act_day": 'act_min'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    result_data['actmin_red_register'] = result_data['act_min'] - result_data['register_day']
    result_data['maxday_red_actmin'] = max(result_data['register_day']) - result_data['act_min']
    result_data = result_data.fillna(-1)

    # 最后一天是否有活动
    result_data['act_at_lastday'] = pd.Series(result_data['act_max'] == max(feature_data['act_day'])).replace(
        {True: 1, False: 0})

    # 均值/最大/最小 天数处理
    result_data['act_mean'] = max(feature_data['act_day']) - result_data['act_mean']
    result_data['act_max'] = max(feature_data['act_day']) - result_data['act_max']
    result_data['act_min'] = max(feature_data['act_day']) - result_data['act_min']

    # 观看自己计数
    feature_tmp = pd.pivot_table(feature_data[feature_data['user_id'] == feature_data['author_id']],
                                 index='user_id', values='author_id', aggfunc='count').reset_index().rename(
        columns={"author_id": 'act_self_count'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    result_data = result_data.fillna(0)

    # 活动日期的 方差/峰度/偏度
    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='act_day',
                                 aggfunc=np.var).reset_index().rename(columns={"act_day": 'act_var'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='act_day',
                                 aggfunc=pd.Series.kurt).reset_index().rename(columns={"act_day": 'act_kurt'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    feature_tmp = pd.pivot_table(feature_data, index='user_id', values='act_day',
                                 aggfunc=pd.Series.skew).reset_index().rename(columns={"act_day": 'act_skew'})
    result_data = pd.merge(result_data, feature_tmp, on='user_id', how='left')
    result_data = result_data.fillna(0)

    # action 的 计数/率
    feature_tmp = feature_data.loc[:, ['user_id', 'action_type', 'act_day']].groupby(
        ['user_id', 'action_type']).count().reset_index().rename(columns={"act_day": 'action_count'})
    for i in range(6):
        fea_name = 'action_' + str(i) + '_count'
        action_tmp = feature_tmp[feature_tmp['action_type'] == i].loc[:, ['user_id', 'action_count']].rename(
            columns={"action_count": fea_name})
        result_data = pd.merge(result_data, action_tmp, how='left', on='user_id')
    result_data = result_data.fillna(0)
    result_data['action_all'] = (result_data['action_0_count'] + result_data['action_1_count'] +
                                 result_data['action_2_count'] + result_data['action_3_count'] +
                                 result_data['action_4_count'] + result_data['action_5_count']).replace(0, 1)
    for i in range(6):
        fea_name = 'action_' + str(i) + '_ratio'
        fea_name_2 = 'action_' + str(i) + '_count'
        result_data[fea_name] = result_data[fea_name_2] / result_data['action_all']

    # page 的 计数/率
    feature_tmp = feature_data.loc[:, ['user_id', 'page', 'act_day']].groupby(
        ['user_id', 'page']).count().reset_index().rename(columns={"act_day": 'page_count'})
    for i in range(5):
        fea_name = 'page_' + str(i) + '_count'
        page_tmp = feature_tmp[feature_tmp['page'] == i].loc[:, ['user_id', 'page_count']].rename(
            columns={"page_count": fea_name})
        result_data = pd.merge(result_data, page_tmp, how='left', on='user_id')
    result_data = result_data.fillna(0)
    result_data['page_all'] = (result_data['page_0_count'] + result_data['page_1_count'] +
                               result_data['page_2_count'] + result_data['page_3_count'] +
                               result_data['page_4_count']).replace(0, 1)
    for i in range(5):
        fea_name = 'page_' + str(i) + '_ratio'
        fea_name_2 = 'page_' + str(i) + '_count'
        result_data[fea_name] = result_data[fea_name_2] / result_data['page_all']

    del result_data['page_all']
    del result_data['action_all']
    del result_data['register_day']

    # 保存结果
    result_file_name = 'activity_feature_' + str(feature_num) + '.csv'
    result_data.to_csv('../data/feature/' + result_file_name, index=None)

